Model Backtesting

Code

import eia_backtesting
import pandas as pd
import plotly.graph_objects as go
import plotly.express as px
import os
import datetime
import requests
import json

Support for Torch based models not available. To enable them, install "darts", "u8darts[torch]" or "u8darts[all]" (with pip); or "u8darts-torch" or "u8darts-all" (with conda).
/opt/ai_dev_workshop/lib/python3.10/site-packages/statsforecast/core.py:27: TqdmExperimentalWarning:

Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)

Load data

Code

raw_json = open("../settings/series.json")
meta_json = json.load(raw_json)

meta_path = meta_json["meta_path"]
data_path = meta_json["data_path"]
leaderboard_path = meta_json["leaderboard_path"]

input = pd.read_csv(data_path)

input["period"] = pd.to_datetime(input["period"])

end = input.groupby(['subba'])['period'].max().min().floor(freq = "d") - datetime.timedelta(hours = 1)

print(end)


input.head()

2024-06-14 23:00:00

	period	subba	subba-name	parent	parent-name	value	value-units
0	2018-07-01 08:00:00	PGAE	Pacific Gas and Electric	CISO	California Independent System Operator	12522.0	megawatthours
1	2018-07-01 09:00:00	PGAE	Pacific Gas and Electric	CISO	California Independent System Operator	11745.0	megawatthours
2	2018-07-01 10:00:00	PGAE	Pacific Gas and Electric	CISO	California Independent System Operator	11200.0	megawatthours
3	2018-07-01 11:00:00	PGAE	Pacific Gas and Electric	CISO	California Independent System Operator	10822.0	megawatthours
4	2018-07-01 12:00:00	PGAE	Pacific Gas and Electric	CISO	California Independent System Operator	10644.0	megawatthours

Code

p = px.line(input, x="period", y="value", color="subba",
 labels={
                     "period": "Time",
                     "value": "MegaWattHours",
                     "subba": "Sub-Region"
                 },
title = "California Hourly Demand By Operating Provider")
p.show()

Code

freq = meta_json["backtesting"]["freq"]
h = meta_json["backtesting"]["h"]
overlap = meta_json["backtesting"]["overlap"]
tags = meta_json["backtesting"]["tags"]
experiment_name = meta_json["backtesting"]["experiment_name"]
mlflow_path = meta_json["backtesting"]["mlflow_path"]
p = meta_json["backtesting"]["p"]
pi = meta_json["backtesting"]["pi"]
quantiles = meta_json["backtesting"]["quantiles"]
seed = meta_json["backtesting"]["seed"]


params1 = meta_json["backtesting"]["models"]["model1"]
params2 = meta_json["backtesting"]["models"]["model2"]
params3 = meta_json["backtesting"]["models"]["model3"]
params4 = meta_json["backtesting"]["models"]["model4"]
params5 = meta_json["backtesting"]["models"]["model5"]
params6 = meta_json["backtesting"]["models"]["model6"]
params7 = meta_json["backtesting"]["models"]["model7"]

params = [params1, params2, params3, params4, params5, params7]

for i in range(len(params)):
    params[i]["h"] = h
    params[i]["freq"] = freq
    params[i]["quantiles"] = quantiles
    params[i]["pi"] = pi
    params[i]["seed"] = seed

Code

series = pd.DataFrame(meta_json["series"])

backtesting_output = {}
for index, row in series.iterrows():
    print("Start backtesting for " + row['subba_id'])
    df = None
    df = input[input["subba"] == row['subba_id']]
    df = df.sort_values(by = ["period"])
    par = params
    for i in range(len(par)):
        par[i]["subba"] = row['subba_id']

    backtesting_output[row['subba_id']] = eia_backtesting.backtesting(input = df, 
            partitions=p, 
            overlap = overlap, 
            h = h, 
            params = par,
            experiment_name = experiment_name + "_" + row['subba_id'],
            mlflow_path = mlflow_path,
            overwrite = True,
            tags = tags)

Code

leaderboard = None
best = None

for index, row in series.iterrows():
    subba = row["subba_id"]
    print(subba)
    print(backtesting_output[subba].leaderboard)
    leader_subba = backtesting_output[subba].leaderboard
    leader_subba["subba"] = subba

    if leaderboard is None:
        leaderboard = backtesting_output[subba].leaderboard
        best =  leader_subba[leader_subba["mape"] == leader_subba["mape"].min()]
    else:
        leaderboard_temp = backtesting_output[subba].leaderboard
        leaderboard = leaderboard._append(leaderboard_temp)
        best = best._append(leader_subba[leader_subba["mape"] == leader_subba["mape"].min()])

    
#best = best[["subba", "model","model_label" ,"mape", "rmse"]] 
best.to_csv(leaderboard_path, index = False)   

best

PGAE
  forecast_label model_label                  model      mape         rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.079842  1135.350522   
1     2024-06-14      model2  LinearRegressionModel  0.080266  1141.392395   
2     2024-06-14      model3  LinearRegressionModel  0.057233   816.260224   
3     2024-06-14      model4  LinearRegressionModel  0.061586   900.585172   
4     2024-06-14      model5  LinearRegressionModel  0.084842  1216.658125   
5     2024-06-14      model7               XGBModel  0.085270  1180.609964   

   coverage                                           comments  
0  0.808333  LM model with lags, training with 2 years of h...  
1  0.804167  LM model with lags, training with 3 years of h...  
2  0.829167                                 Model 2 with lag 1  
3  0.808333                       Model 1 with additional lags  
4  0.789583                       Model 1 with additional lags  
5  0.733333                                  XGBoost with lags  
SCE
  forecast_label model_label                  model      mape        rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.053219  776.039709   
1     2024-06-14      model2  LinearRegressionModel  0.053042  772.618437   
2     2024-06-14      model3  LinearRegressionModel  0.046309  655.991958   
3     2024-06-14      model4  LinearRegressionModel  0.044686  653.597728   
4     2024-06-14      model5  LinearRegressionModel  0.068357  985.158385   
5     2024-06-14      model7               XGBModel  0.069670  970.892884   

   coverage                                           comments  
0  0.929167  LM model with lags, training with 2 years of h...  
1  0.929167  LM model with lags, training with 3 years of h...  
2  0.889583                                 Model 2 with lag 1  
3  0.904167                       Model 1 with additional lags  
4  0.893750                       Model 1 with additional lags  
5  0.789583                                  XGBoost with lags  
SDGE
  forecast_label model_label                  model      mape        rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.083208  173.564459   
1     2024-06-14      model2  LinearRegressionModel  0.082176  173.083759   
2     2024-06-14      model3  LinearRegressionModel  0.110862  206.756412   
3     2024-06-14      model4  LinearRegressionModel  0.098167  189.345107   
4     2024-06-14      model5  LinearRegressionModel  0.125380  235.072625   
5     2024-06-14      model7               XGBModel  0.146632  269.623860   

   coverage                                           comments  
0  0.958333  LM model with lags, training with 2 years of h...  
1  0.956250  LM model with lags, training with 3 years of h...  
2  0.889583                                 Model 2 with lag 1  
3  0.925000                       Model 1 with additional lags  
4  0.889583                       Model 1 with additional lags  
5  0.858333                                  XGBoost with lags  
VEA
  forecast_label model_label                  model      mape       rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.105364  18.354512   
1     2024-06-14      model2  LinearRegressionModel  0.105227  18.476149   
2     2024-06-14      model3  LinearRegressionModel  0.155049  26.586133   
3     2024-06-14      model4  LinearRegressionModel  0.137781  22.932522   
4     2024-06-14      model5  LinearRegressionModel  0.134385  23.258212   
5     2024-06-14      model7               XGBModel  0.115111  21.245936   

   coverage                                           comments  
0  0.895833  LM model with lags, training with 2 years of h...  
1  0.841667  LM model with lags, training with 3 years of h...  
2  0.797917                                 Model 2 with lag 1  
3  0.820833                       Model 1 with additional lags  
4  0.854167                       Model 1 with additional lags  
5  0.879167                                  XGBoost with lags

	forecast_label	model_label	model	mape	rmse	coverage	comments	subba
2	2024-06-14	model3	LinearRegressionModel	0.057233	816.260224	0.829167	Model 2 with lag 1	PGAE
3	2024-06-14	model4	LinearRegressionModel	0.044686	653.597728	0.904167	Model 1 with additional lags	SCE
1	2024-06-14	model2	LinearRegressionModel	0.082176	173.083759	0.956250	LM model with lags, training with 3 years of h...	SDGE
1	2024-06-14	model2	LinearRegressionModel	0.105227	18.476149	0.841667	LM model with lags, training with 3 years of h...	VEA

Code

print(leaderboard)

  forecast_label model_label                  model      mape         rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.079842  1135.350522   
1     2024-06-14      model2  LinearRegressionModel  0.080266  1141.392395   
2     2024-06-14      model3  LinearRegressionModel  0.057233   816.260224   
3     2024-06-14      model4  LinearRegressionModel  0.061586   900.585172   
4     2024-06-14      model5  LinearRegressionModel  0.084842  1216.658125   
5     2024-06-14      model7               XGBModel  0.085270  1180.609964   
0     2024-06-14      model1  LinearRegressionModel  0.053219   776.039709   
1     2024-06-14      model2  LinearRegressionModel  0.053042   772.618437   
2     2024-06-14      model3  LinearRegressionModel  0.046309   655.991958   
3     2024-06-14      model4  LinearRegressionModel  0.044686   653.597728   
4     2024-06-14      model5  LinearRegressionModel  0.068357   985.158385   
5     2024-06-14      model7               XGBModel  0.069670   970.892884   
0     2024-06-14      model1  LinearRegressionModel  0.083208   173.564459   
1     2024-06-14      model2  LinearRegressionModel  0.082176   173.083759   
2     2024-06-14      model3  LinearRegressionModel  0.110862   206.756412   
3     2024-06-14      model4  LinearRegressionModel  0.098167   189.345107   
4     2024-06-14      model5  LinearRegressionModel  0.125380   235.072625   
5     2024-06-14      model7               XGBModel  0.146632   269.623860   
0     2024-06-14      model1  LinearRegressionModel  0.105364    18.354512   
1     2024-06-14      model2  LinearRegressionModel  0.105227    18.476149   
2     2024-06-14      model3  LinearRegressionModel  0.155049    26.586133   
3     2024-06-14      model4  LinearRegressionModel  0.137781    22.932522   
4     2024-06-14      model5  LinearRegressionModel  0.134385    23.258212   
5     2024-06-14      model7               XGBModel  0.115111    21.245936   

   coverage                                           comments subba  
0  0.808333  LM model with lags, training with 2 years of h...  PGAE  
1  0.804167  LM model with lags, training with 3 years of h...  PGAE  
2  0.829167                                 Model 2 with lag 1  PGAE  
3  0.808333                       Model 1 with additional lags  PGAE  
4  0.789583                       Model 1 with additional lags  PGAE  
5  0.733333                                  XGBoost with lags  PGAE  
0  0.929167  LM model with lags, training with 2 years of h...   SCE  
1  0.929167  LM model with lags, training with 3 years of h...   SCE  
2  0.889583                                 Model 2 with lag 1   SCE  
3  0.904167                       Model 1 with additional lags   SCE  
4  0.893750                       Model 1 with additional lags   SCE  
5  0.789583                                  XGBoost with lags   SCE  
0  0.958333  LM model with lags, training with 2 years of h...  SDGE  
1  0.956250  LM model with lags, training with 3 years of h...  SDGE  
2  0.889583                                 Model 2 with lag 1  SDGE  
3  0.925000                       Model 1 with additional lags  SDGE  
4  0.889583                       Model 1 with additional lags  SDGE  
5  0.858333                                  XGBoost with lags  SDGE  
0  0.895833  LM model with lags, training with 2 years of h...   VEA  
1  0.841667  LM model with lags, training with 3 years of h...   VEA  
2  0.797917                                 Model 2 with lag 1   VEA  
3  0.820833                       Model 1 with additional lags   VEA  
4  0.854167                       Model 1 with additional lags   VEA  
5  0.879167                                  XGBoost with lags   VEA

PGAE

Code

 eia_backtesting.plot_score(score = backtesting_output["PGAE"].leaderboard, type = "box")

Code

eia_backtesting.plot_score(score = backtesting_output["PGAE"].score, type = "line")

SCE

Code

 eia_backtesting.plot_score(score = backtesting_output["SCE"].leaderboard, type = "box")

Code

eia_backtesting.plot_score(score = backtesting_output["SCE"].score, type = "line")

SDGE

Code

 eia_backtesting.plot_score(score = backtesting_output["SDGE"].leaderboard, type = "box")

Code

eia_backtesting.plot_score(score = backtesting_output["SDGE"].score, type = "line")

VEA

Code

 eia_backtesting.plot_score(score = backtesting_output["VEA"].leaderboard, type = "box")

Code

eia_backtesting.plot_score(score = backtesting_output["VEA"].score, type = "line")